{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from brnn_model_2 import *\n",
    "import reader\n",
    "\n",
    "import subprocess\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Global variables\n",
    "\"\"\"\n",
    "model_type = \"test\"\n",
    "data_path = \"../data/\"\n",
    "save_path = \"./saved_model\"\n",
    "global_prior_pi = 0.25\n",
    "global_log_sigma1 = -1.0\n",
    "global_log_sigma2 = -7.0\n",
    "global_random_seed = 12\n",
    "global_num_gpus = 0\n",
    "\n",
    "\n",
    "# Model can be \"test\", \"small\", \"medium\", \"large\"\n",
    "model_select = \"test\"\n",
    "\n",
    "#Put the path to the data here\n",
    "dat_path = \"../data\"\n",
    "\n",
    "#Put the path to where you want to save the training data\n",
    "sav_path = \"tensorboard/\"\n",
    "\n",
    "# The mixing degree for the prior gaussian mixture\n",
    "# As in Fortunato they report scanning\n",
    "# mix_pi \\in { 1/4, 1/2, 3/4 }\n",
    "mixing_pi = 0.25\n",
    "\n",
    "# As in Fortunato they report scanning\n",
    "# log sigma1 \\in { 0, -1, -2 }\n",
    "# log sigma2 \\in { -6, -7, -8 }\n",
    "prior_log_sigma1 = -1.0\n",
    "prior_log_sigma2 = -7.0\n",
    "\n",
    "\n",
    "class SmallConfig(object):\n",
    "    \"\"\"Small config.\"\"\"\n",
    "    init_scale = 0.1\n",
    "    learning_rate = 1.0\n",
    "    max_grad_norm = 5\n",
    "    num_layers = 2\n",
    "    num_steps = 20\n",
    "    hidden_size = 200\n",
    "    max_epoch = 4\n",
    "    max_max_epoch = 13\n",
    "    keep_prob = 1.0\n",
    "    lr_decay = 0.5\n",
    "    \n",
    "    batch_size = 20\n",
    "    vocab_size = 10000\n",
    "    \n",
    "    X_dim = 200 # Size of the embedding\n",
    "\n",
    "class MediumConfig(object):\n",
    "    \"\"\"\n",
    "    Medium config.\n",
    "    Slightly modified according to email.\n",
    "    \"\"\"\n",
    "    init_scale = 0.05\n",
    "    learning_rate = 1.0\n",
    "    max_grad_norm = 5\n",
    "    num_layers = 2\n",
    "    num_steps = 35\n",
    "    hidden_size = 650\n",
    "    max_epoch = 20\n",
    "    max_max_epoch = 70\n",
    "    keep_prob = 1.0\n",
    "    lr_decay = 0.9\n",
    "    batch_size = 20\n",
    "    vocab_size = 10000\n",
    "\n",
    "    X_dim = 50 # Size of the embedding\n",
    "    \n",
    "class LargeConfig(object):\n",
    "    \"\"\"Large config.\"\"\"\n",
    "    init_scale = 0.04\n",
    "    learning_rate = 1.0\n",
    "    max_grad_norm = 10\n",
    "    num_layers = 2\n",
    "    num_steps = 35\n",
    "    hidden_size = 1500\n",
    "    max_epoch = 14\n",
    "    max_max_epoch = 55\n",
    "    keep_prob = 0.35\n",
    "    lr_decay = 1 / 1.15\n",
    "    batch_size = 20\n",
    "    vocab_size = 10000\n",
    "\n",
    "    X_dim = 100 # Size of the embedding\n",
    "    \n",
    "class TestConfig(object):\n",
    "    \"\"\"Tiny config, for testing.\"\"\"\n",
    "    init_scale = 0.1\n",
    "    learning_rate = 1.0\n",
    "    max_grad_norm = 1\n",
    "    num_layers = 2\n",
    "    num_steps = 20\n",
    "    hidden_size = 15\n",
    "    max_epoch = 1\n",
    "    max_max_epoch = 1\n",
    "    keep_prob = 1.0\n",
    "    lr_decay = 0.5\n",
    "    batch_size = 20\n",
    "    vocab_size = 10000\n",
    "\n",
    "    X_dim = 19 # Size of the embedding\n",
    "\n",
    "\n",
    "#    global_random_seed = set_random_seed\n",
    "    \n",
    "def get_config():\n",
    "    \"\"\"Get model config.\"\"\"\n",
    "    if model_type == \"small\":\n",
    "        config = SmallConfig()\n",
    "    elif model_type == \"medium\":\n",
    "        config = MediumConfig()\n",
    "    elif model_type == \"large\":\n",
    "        config = LargeConfig()\n",
    "    elif model_type == \"test\":\n",
    "        config = TestConfig()\n",
    "    else:\n",
    "        raise ValueError(\"Invalid model: %s\", model_type)\n",
    "\n",
    "    print (\"Model Type\")\n",
    "    print (model_type)\n",
    "    config.prior_pi = global_prior_pi\n",
    "    config.log_sigma1 = global_log_sigma1\n",
    "    config.log_sigma2 = global_log_sigma2\n",
    "\n",
    "    return config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test\n",
      "Model Type\n",
      "test\n",
      "Model Type\n",
      "test\n",
      "INFO:tensorflow:Summary name KL Loss is illegal; using KL_Loss instead.\n",
      "INFO:tensorflow:Summary name Total Loss is illegal; using Total_Loss instead.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#    change_random_seed(global_random_seed)\n",
    "raw_data = reader.ptb_raw_data(data_path)\n",
    "train_data, valid_data, test_data, _ = raw_data\n",
    "\n",
    "print (model_type)\n",
    "\n",
    "config = get_config()\n",
    "eval_config = get_config()\n",
    "#eval_config.batch_size = 1\n",
    "#eval_config.num_steps = 1\n",
    "\n",
    "#subprocess.Popen([\"tensorboard\",\"--logdir=tensorboard\"])\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    initializer = tf.random_uniform_initializer(-config.init_scale,\n",
    "                                                config.init_scale)\n",
    "\n",
    "    with tf.name_scope(\"Train\"):\n",
    "        train_input = PTBInput(config=config, data=train_data, name=\"TrainInput\")\n",
    "        with tf.variable_scope(\"Model\", reuse=None, initializer=initializer):\n",
    "            m = PTBModel(is_training=True, config=config, input_=train_input)\n",
    "        tf.summary.scalar(\"Training_Loss\", m.cost)\n",
    "        tf.summary.scalar(\"Learning_Rate\", m.lr)\n",
    "        tf.summary.scalar(\"KL Loss\", m.kl_loss)\n",
    "        tf.summary.scalar(\"Total Loss\", m.total_loss)\n",
    "\n",
    "    with tf.name_scope(\"Valid\"):\n",
    "        valid_input = PTBInput(config=config, data=valid_data, name=\"ValidInput\")\n",
    "        with tf.variable_scope(\"Model\", reuse=True, initializer=initializer):\n",
    "            mvalid = PTBModel(is_training=False, config=config, input_=valid_input)\n",
    "        tf.summary.scalar(\"Validation_Loss\", mvalid.cost)\n",
    "\n",
    "    with tf.name_scope(\"Test\"):\n",
    "        test_input = PTBInput(\n",
    "            config=eval_config, data=test_data, name=\"TestInput\")\n",
    "        with tf.variable_scope(\"Model\", reuse=True, initializer=initializer):\n",
    "            mtest = PTBModel(is_training=False, config=eval_config,\n",
    "                             input_=test_input)\n",
    "\n",
    "    models = {\"Train\": m, \"Valid\": mvalid, \"Test\": mtest}\n",
    "    for name, model in models.items():\n",
    "        model.export_ops(name)\n",
    "    metagraph = tf.train.export_meta_graph()\n",
    "    soft_placement = False\n",
    "    if global_num_gpus > 1:\n",
    "        soft_placement = True\n",
    "        util.auto_parallel(metagraph, m)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./saved_model/model.ckpt-0\n",
      "INFO:tensorflow:Starting standard services.\n",
      "INFO:tensorflow:Saving checkpoint to path ./saved_model/model.ckpt\n",
      "INFO:tensorflow:Starting queue runners.\n",
      "INFO:tensorflow:Model/global_step/sec: 0\n",
      "INFO:tensorflow:Recording summary at step 0.\n",
      "Epoch: 1 Learning rate: 1.000\n",
      "0.000 perplexity: 10068.906 speed: 669 wps\n",
      "KL is 9.608443260192871\n",
      "0.004 perplexity: 5282.180 speed: 2906 wps\n",
      "KL is 9.613214492797852\n",
      "0.104 perplexity: 981.084 speed: 4262 wps\n",
      "KL is 9.649154663085938\n",
      "0.204 perplexity: 884.661 speed: 4303 wps\n",
      "KL is 9.665657043457031\n",
      "0.304 perplexity: 840.052 speed: 4174 wps\n",
      "KL is 9.659225463867188\n",
      "0.404 perplexity: 816.887 speed: 4160 wps\n",
      "KL is 9.678943634033203\n",
      "0.504 perplexity: 795.834 speed: 3981 wps\n",
      "KL is 9.69120979309082\n",
      "INFO:tensorflow:Model/global_step/sec: 9.92781\n",
      "INFO:tensorflow:Recording summary at step 1192.\n",
      "0.604 perplexity: 759.520 speed: 3970 wps\n",
      "KL is 9.7293062210083\n",
      "0.703 perplexity: 725.680 speed: 3915 wps\n",
      "KL is 9.72191047668457\n",
      "0.803 perplexity: 694.858 speed: 3871 wps\n",
      "KL is 9.766129493713379\n",
      "0.903 perplexity: 664.656 speed: 3836 wps\n",
      "KL is 9.795852661132812\n",
      "INFO:tensorflow:Model/global_step/sec: 9.06695\n",
      "INFO:tensorflow:Recording summary at step 2280.\n",
      "Epoch: 1 Train Perplexity: 638.712\n",
      "Epoch: 1 Valid Perplexity: 427.946\n",
      "Test Perplexity: 413.327\n",
      "Saving model to ./saved_model.\n",
      "----------------------------------------------------------------\n",
      "------------------ Prediction of Sentences ---------------------\n",
      "Computing batch 0/206\n"
     ]
    }
   ],
   "source": [
    "\n",
    "## Training !\n",
    "with tf.Graph().as_default():\n",
    "    tf.train.import_meta_graph(metagraph)\n",
    "    for model in models.values():\n",
    "        model.import_ops()\n",
    "    sv = tf.train.Supervisor(logdir=save_path)\n",
    "    config_proto = tf.ConfigProto(allow_soft_placement=soft_placement)\n",
    "    with sv.managed_session(config=config_proto) as session:\n",
    "\n",
    "        for i in range(config.max_max_epoch):\n",
    "            lr_decay = config.lr_decay ** max(i + 1 - config.max_epoch, 0.0)\n",
    "            m.assign_lr(session, config.learning_rate * lr_decay)\n",
    "\n",
    "            print(\"Epoch: %d Learning rate: %.3f\" % (i + 1, session.run(m.lr)))\n",
    "            train_perplexity = run_epoch(session, m, eval_op=m.train_op,\n",
    "                                         verbose=True)\n",
    "            print(\"Epoch: %d Train Perplexity: %.3f\" % (i + 1, train_perplexity))\n",
    "            valid_perplexity = run_epoch(session, mvalid)\n",
    "            print(\"Epoch: %d Valid Perplexity: %.3f\" % (i + 1, valid_perplexity))\n",
    "            \n",
    "        test_perplexity = run_epoch(session, mtest)\n",
    "        print(\"Test Perplexity: %.3f\" % test_perplexity)\n",
    "        \n",
    "\n",
    "        print(\"Saving model to %s.\" % save_path)\n",
    "        sv.saver.save(session, save_path, global_step=sv.global_step)\n",
    "    \n",
    "        print (\"----------------------------------------------------------------\")\n",
    "        print (\"------------------ Prediction of Sentences ---------------------\")\n",
    "\n",
    "       #  inputs, predicted = fetch_output(session, mtest)\n",
    "\n",
    "        costs = 0.0\n",
    "        state = session.run(model.initial_state)\n",
    "\n",
    "        inputs = []\n",
    "        outputs = []\n",
    "        fetches = {\n",
    "            \"final_state\": model.final_state,\n",
    "            \"output\": model.output,\n",
    "            \"input\": model.input_data\n",
    "        }\n",
    "\n",
    "        for step in range(model.input.epoch_size):\n",
    "            feed_dict = {}\n",
    "            for i, (c, h) in enumerate(model.initial_state):\n",
    "                feed_dict[c] = state[i].c\n",
    "                feed_dict[h] = state[i].h\n",
    "\n",
    "            print (\"Computing batch %i/%i\"%(step, model.input.epoch_size))\n",
    "            vals = session.run(fetches, feed_dict)\n",
    "            state = vals[\"final_state\"]\n",
    "            output = vals[\"output\"]\n",
    "            input_i = vals[\"input\"]\n",
    "            outputs.append(output)\n",
    "            inputs.append(input_i)\n",
    "            \n",
    "            break;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing\n",
      "INFO:tensorflow:Restoring parameters from ./saved_model/model.ckpt-0\n",
      "INFO:tensorflow:Starting standard services.\n",
      "INFO:tensorflow:Saving checkpoint to path ./saved_model/model.ckpt\n",
      "INFO:tensorflow:Starting queue runners.\n",
      "INFO:tensorflow:Recording summary at step 0.\n",
      "Test Perplexity: 10029.646\n",
      "----------------------------------------------------------------\n",
      "------------------ Prediction of Sentences ---------------------\n",
      "Computing batch 0/206\n"
     ]
    }
   ],
   "source": [
    "## Testing\n",
    "\"\"\"\n",
    "print (\"Testing\")\n",
    "predicted = []   # Variable to store predictions\n",
    "with tf.Graph().as_default():\n",
    "    #tf.train.import_meta_graph(metagraph)\n",
    "    #for model in models.values():\n",
    "    #    model.import_ops()\n",
    "    #sv = tf.train.Supervisor(logdir=save_path)\n",
    "    #config_proto = tf.ConfigProto(allow_soft_placement=soft_placement)\n",
    "    with sv.managed_session(config=config_proto) as session:\n",
    "        \n",
    "       # session = tf.Session()\n",
    "    \n",
    "        test_perplexity = run_epoch(session, mtest)\n",
    "        print(\"Test Perplexity: %.3f\" % test_perplexity)\n",
    "\n",
    "        print (\"----------------------------------------------------------------\")\n",
    "        print (\"------------------ Prediction of Sentences ---------------------\")\n",
    "\n",
    "       #  inputs, predicted = fetch_output(session, mtest)\n",
    "\n",
    "        costs = 0.0\n",
    "        state = session.run(model.initial_state)\n",
    "\n",
    "        inputs = []\n",
    "        outputs = []\n",
    "        fetches = {\n",
    "            \"final_state\": model.final_state,\n",
    "            \"output\": model.output,\n",
    "            \"input\": model.input_data\n",
    "        }\n",
    "\n",
    "        for step in range(model.input.epoch_size):\n",
    "            feed_dict = {}\n",
    "            for i, (c, h) in enumerate(model.initial_state):\n",
    "                feed_dict[c] = state[i].c\n",
    "                feed_dict[h] = state[i].h\n",
    "\n",
    "            print (\"Computing batch %i/%i\"%(step, model.input.epoch_size))\n",
    "            vals = session.run(fetches, feed_dict)\n",
    "            state = vals[\"final_state\"]\n",
    "            output = vals[\"output\"]\n",
    "            input_i = vals[\"input\"]\n",
    "            outputs.append(output)\n",
    "            inputs.append(input_i)\n",
    "            \n",
    "            break;\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input and output of the first chain of the first batch\n",
      "[ 102   14   24   32  752  381    2   29  120    0   35   92   60  111  143\n",
      "   32  616 3148  282   19]\n",
      "[[  1.88022375e-03   2.35194527e-02   9.97530879e-04 ...,   7.36053626e-05\n",
      "    7.02992256e-05   7.01820318e-05]\n",
      " [  1.34575646e-03   3.31664868e-02   5.28996810e-03 ...,   4.77102731e-05\n",
      "    4.94470478e-05   4.25421385e-05]\n",
      " [  1.16743688e-02   6.29757643e-02   5.28007187e-03 ...,   3.75037598e-05\n",
      "    3.74578813e-05   3.53951436e-05]\n",
      " ..., \n",
      " [  3.83780599e-02   3.40645909e-02   1.11804068e-01 ...,   8.83910798e-06\n",
      "    8.94827372e-06   7.23520861e-06]\n",
      " [  3.19442414e-02   3.54684144e-02   1.00499868e-01 ...,   8.59345437e-06\n",
      "    8.90955198e-06   6.98457688e-06]\n",
      " [  1.46348223e-01   6.69299141e-02   4.62764036e-03 ...,   1.38146179e-05\n",
      "    1.25355537e-05   1.23736636e-05]]\n"
     ]
    }
   ],
   "source": [
    "print (\"Input and output of the first chain of the first batch\")\n",
    "print (inputs[0][0])\n",
    "print (outputs[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 10000)\n",
      "[1 1 1 1 1 5 0 0 0 1 1 1 9 1 1 1 5 2 2 0]\n"
     ]
    }
   ],
   "source": [
    "selected_words = np.argmax(outputs[0][0], axis = 1)\n",
    "print (outputs[0][0].shape)\n",
    "print (selected_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
